From 219f9902c9fa8016934b2617506e66f671277534 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 31 Aug 2014 23:03:45 -0700 Subject: [PATCH] Implement git authentication This commit updates git2-rs to get the implementation of the authentication callback in libgit2. Additionally this specifies the callback for whenever we're cloning into the database or updating submodules. Currently cargo will *not* ask for user input, but rather require you to have authentication configured in git through some other means. There are currently two primary methods of doing so: 1. Any SSH key in the local ssh-agent will be used for authentication with SSH repositories. 2. The `credential.helper` interface (as specified by gitcredential(7)) has been implemented in git2-rs to allow for picking up of storage of passwords in the local git cache or keychain. If these two methods fail, then there will likely be an authentication failure. Interactive prompts for authentication have not been implemented as there is no method to currently enter your password into the terminal silently. A consequence of this commit is that cargo now depends on libssh2. A package was created to create a static copy of libssh2, and this is now linked into cargo by default. It turned out that just building libssh2 was quite a beast in and of itself on windows. The primary stickler point is that on the current release, 1.4.3, libssh2 requires openssl on windows. At this time I don't want to pick up a dependency on openssl for windows, and it turned out that the unreleased 1.4.4 version has a new backend for windows not based on openssl, but rather windows's cryptography API. The current bundled version of libssh2 is 1.4.4 with some light modifications to actually build on windows (wow that was hard). All in all, we're now statically linking to libssh 1.4.4 (not a runtime dependency). Closes #493 --- Cargo.lock | 57 +++++---- Makefile.in | 2 +- src/cargo/sources/git/utils.rs | 88 +++++++++++-- src/etc/print-new-snapshot.py | 2 +- src/snapshots.txt | 7 + tests/support/mod.rs | 1 - tests/test_cargo_build_auth.rs | 226 +++++++++++++++++++++++++++++++++ tests/tests.rs | 4 +- 8 files changed, 345 insertions(+), 42 deletions(-) create mode 100644 tests/test_cargo_build_auth.rs diff --git a/Cargo.lock b/Cargo.lock index e4882091c..feb05d64a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,34 +2,34 @@ name = "cargo" version = "0.0.1-pre" dependencies = [ - "docopt 0.6.2 (git+https://github.com/burntsushi/docopt.rs#3bf52a9f9cf13e00cca00ff49da39d5c9caa48e9)", - "docopt_macros 0.6.2 (git+https://github.com/burntsushi/docopt.rs#3bf52a9f9cf13e00cca00ff49da39d5c9caa48e9)", - "flate2 0.0.1 (git+https://github.com/alexcrichton/flate2-rs#12593d1b9ccf09c2eabac176a6e233b171eed843)", - "git2 0.0.1 (git+https://github.com/alexcrichton/git2-rs#f2b79dd951e92171b151100d69c32c0bf137a328)", + "docopt 0.6.3 (git+https://github.com/burntsushi/docopt.rs#652c165c6c05629dee8a19af6c62103082582a99)", + "docopt_macros 0.6.3 (git+https://github.com/burntsushi/docopt.rs#652c165c6c05629dee8a19af6c62103082582a99)", + "flate2 0.0.1 (git+https://github.com/alexcrichton/flate2-rs#2ccf4dc3cb613446c6f80fbe1d6950875417de29)", + "git2 0.0.1 (git+https://github.com/alexcrichton/git2-rs#e26e6d635f74f02e4c627d07daaaf2a1483e7b10)", "glob 0.0.1 (git+https://github.com/rust-lang/glob#c4495d9f2f2a1b22173b860f907760ba8c419843)", "hamcrest 0.1.0 (git+https://github.com/carllerche/hamcrest-rust.git#f0fd1546b0a7a278a12658ab8602b5c827cc3a42)", - "semver 0.0.1 (git+https://github.com/rust-lang/semver#c78b40d7fdf8acd99b503e6ce394fbcf9eb8982f)", - "tar 0.0.1 (git+https://github.com/alexcrichton/tar-rs#d4ce3448a1a229b78f16d31682140c2843479481)", - "toml 0.1.0 (git+https://github.com/alexcrichton/toml-rs#e3ce3517348dd5f6f6306dc1f3bae7f9dd77c0fe)", - "url 0.1.0 (git+https://github.com/servo/rust-url#bb68de835ad945a72fba44979944a587ba83941a)", + "semver 0.0.1 (git+https://github.com/rust-lang/semver#df163f7b22686493b037eee1f1f9d1a2742f9bbe)", + "tar 0.0.1 (git+https://github.com/alexcrichton/tar-rs#a87a4b9c8087922454a118bee97ecdaa1f8cbc18)", + "toml 0.1.0 (git+https://github.com/alexcrichton/toml-rs#d40724ad2d6516d7b6750515153b4c360d63afe9)", + "url 0.1.0 (git+https://github.com/servo/rust-url#d894135cac4e0085f41ba3a816240b792f9e6154)", ] [[package]] name = "docopt" -version = "0.6.2" -source = "git+https://github.com/burntsushi/docopt.rs#3bf52a9f9cf13e00cca00ff49da39d5c9caa48e9" +version = "0.6.3" +source = "git+https://github.com/burntsushi/docopt.rs#652c165c6c05629dee8a19af6c62103082582a99" [[package]] name = "docopt" -version = "0.6.2" -source = "git+https://github.com/docopt/docopt.rs#3bf52a9f9cf13e00cca00ff49da39d5c9caa48e9" +version = "0.6.3" +source = "git+https://github.com/docopt/docopt.rs#652c165c6c05629dee8a19af6c62103082582a99" [[package]] name = "docopt_macros" -version = "0.6.2" -source = "git+https://github.com/burntsushi/docopt.rs#3bf52a9f9cf13e00cca00ff49da39d5c9caa48e9" +version = "0.6.3" +source = "git+https://github.com/burntsushi/docopt.rs#652c165c6c05629dee8a19af6c62103082582a99" dependencies = [ - "docopt 0.6.2 (git+https://github.com/docopt/docopt.rs#3bf52a9f9cf13e00cca00ff49da39d5c9caa48e9)", + "docopt 0.6.3 (git+https://github.com/docopt/docopt.rs#652c165c6c05629dee8a19af6c62103082582a99)", ] [[package]] @@ -40,14 +40,15 @@ source = "git+https://github.com/lifthrasiir/rust-encoding#35f0d70f65f73ba16f296 [[package]] name = "flate2" version = "0.0.1" -source = "git+https://github.com/alexcrichton/flate2-rs#12593d1b9ccf09c2eabac176a6e233b171eed843" +source = "git+https://github.com/alexcrichton/flate2-rs#2ccf4dc3cb613446c6f80fbe1d6950875417de29" [[package]] name = "git2" version = "0.0.1" -source = "git+https://github.com/alexcrichton/git2-rs#f2b79dd951e92171b151100d69c32c0bf137a328" +source = "git+https://github.com/alexcrichton/git2-rs#e26e6d635f74f02e4c627d07daaaf2a1483e7b10" dependencies = [ - "libgit2 0.0.1 (git+https://github.com/alexcrichton/git2-rs#f2b79dd951e92171b151100d69c32c0bf137a328)", + "libgit2 0.0.1 (git+https://github.com/alexcrichton/git2-rs#e26e6d635f74f02e4c627d07daaaf2a1483e7b10)", + "url 0.1.0 (git+https://github.com/servo/rust-url#d894135cac4e0085f41ba3a816240b792f9e6154)", ] [[package]] @@ -63,12 +64,18 @@ source = "git+https://github.com/carllerche/hamcrest-rust.git#f0fd1546b0a7a278a1 [[package]] name = "libgit2" version = "0.0.1" -source = "git+https://github.com/alexcrichton/git2-rs#f2b79dd951e92171b151100d69c32c0bf137a328" +source = "git+https://github.com/alexcrichton/git2-rs#e26e6d635f74f02e4c627d07daaaf2a1483e7b10" dependencies = [ + "libssh2-static-sys 0.0.1 (git+https://github.com/alexcrichton/libssh2-static-sys#e348c464f94f125cb1491a0ef63b8657012c6b75)", "link-config 0.0.1 (git+https://github.com/alexcrichton/link-config#e378605ce4099008b1dab8f39619d91dc8887946)", - "openssl-static-sys 0.0.1 (git+git://github.com/alexcrichton/openssl-static-sys#b8f2500c39932e9d022dcc2590493ab0cc144e2a)", + "openssl-static-sys 0.0.1 (git+https://github.com/alexcrichton/openssl-static-sys#b8f2500c39932e9d022dcc2590493ab0cc144e2a)", ] +[[package]] +name = "libssh2-static-sys" +version = "0.0.1" +source = "git+https://github.com/alexcrichton/libssh2-static-sys#e348c464f94f125cb1491a0ef63b8657012c6b75" + [[package]] name = "link-config" version = "0.0.1" @@ -77,27 +84,27 @@ source = "git+https://github.com/alexcrichton/link-config#e378605ce4099008b1dab8 [[package]] name = "openssl-static-sys" version = "0.0.1" -source = "git+git://github.com/alexcrichton/openssl-static-sys#b8f2500c39932e9d022dcc2590493ab0cc144e2a" +source = "git+https://github.com/alexcrichton/openssl-static-sys#b8f2500c39932e9d022dcc2590493ab0cc144e2a" [[package]] name = "semver" version = "0.0.1" -source = "git+https://github.com/rust-lang/semver#c78b40d7fdf8acd99b503e6ce394fbcf9eb8982f" +source = "git+https://github.com/rust-lang/semver#df163f7b22686493b037eee1f1f9d1a2742f9bbe" [[package]] name = "tar" version = "0.0.1" -source = "git+https://github.com/alexcrichton/tar-rs#d4ce3448a1a229b78f16d31682140c2843479481" +source = "git+https://github.com/alexcrichton/tar-rs#a87a4b9c8087922454a118bee97ecdaa1f8cbc18" [[package]] name = "toml" version = "0.1.0" -source = "git+https://github.com/alexcrichton/toml-rs#e3ce3517348dd5f6f6306dc1f3bae7f9dd77c0fe" +source = "git+https://github.com/alexcrichton/toml-rs#d40724ad2d6516d7b6750515153b4c360d63afe9" [[package]] name = "url" version = "0.1.0" -source = "git+https://github.com/servo/rust-url#bb68de835ad945a72fba44979944a587ba83941a" +source = "git+https://github.com/servo/rust-url#d894135cac4e0085f41ba3a816240b792f9e6154" dependencies = [ "encoding 0.1.0 (git+https://github.com/lifthrasiir/rust-encoding#35f0d70f65f73ba16f296f9ec675eddee661ba79)", ] diff --git a/Makefile.in b/Makefile.in index cd0bfe47e..b1303b1a5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -64,7 +64,7 @@ all: $(foreach target,$(CFG_TARGET),cargo-$(target)) define CARGO_TARGET cargo-$(1): $$(CARGO) - $$(CFG_RUSTC) -v + "$$(CFG_RUSTC)" -v $$(CARGO) build --target $(1) $$(OPT_FLAG) $$(ARGS) endef $(foreach target,$(CFG_TARGET),$(eval $(call CARGO_TARGET,$(target)))) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index 5694e3dbd..31f65bc3e 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -172,13 +172,9 @@ impl GitRemote { } fn fetch_into(&self, dst: &git2::Repository) -> CargoResult<()> { + // Create a local anonymous remote in the repository to fetch the url let url = self.url.to_string(); - let refspec = "refs/heads/*:refs/heads/*"; - let mut remote = try!(dst.remote_create_anonymous(url.as_slice(), - refspec)); - try!(remote.add_fetch("refs/tags/*:refs/tags/*")); - try!(remote.fetch(None, None)); - Ok(()) + fetch(dst, url.as_slice()) } fn clone_into(&self, dst: &Path) -> CargoResult { @@ -187,10 +183,15 @@ impl GitRemote { try!(rmdir_recursive(dst)); } try!(mkdir_recursive(dst, UserDir)); - let repo = try!(git2::build::RepoBuilder::new().bare(true) - .hardlinks(false) - .clone(url.as_slice(), dst)); - Ok(repo) + let cfg = try!(git2::Config::open_default()); + with_authentication(url.as_slice(), &cfg, |f| { + let repo = try!(git2::build::RepoBuilder::new() + .bare(true) + .hardlinks(false) + .credentials(f) + .clone(url.as_slice(), dst)); + Ok(repo) + }) } } @@ -331,9 +332,7 @@ impl<'a> GitCheckout<'a> { }; // Fetch data from origin and reset to the head commit - let refspec = "refs/heads/*:refs/heads/*"; - let mut remote = try!(repo.remote_create_anonymous(url, refspec)); - try!(remote.fetch(None, None).chain_error(|| { + try!(fetch(&repo, url).chain_error(|| { internal(format!("failed to fetch submodule `{}` from {}", child.name().unwrap_or(""), url)) })); @@ -346,3 +345,66 @@ impl<'a> GitCheckout<'a> { } } } + +fn with_authentication(url: &str, + cfg: &git2::Config, + f: |git2::Credentials| -> CargoResult) + -> CargoResult { + // Prepare the authentication callbacks. + // + // We check the `allowed` types of credentials, and we try to do as much as + // possible based on that: + // + // * Prioritize SSH keys from the local ssh agent as they're likely the most + // reliable. The username here is prioritized from the credential + // callback, then from whatever is configured in git itself, and finally + // we fall back to the generic user of `git`. + // + // * If a username/password is allowed, then we fallback to git2-rs's + // implementation of the credential helper. This is what is configured + // with `credential.helper` in git, and is the interface for the OSX + // keychain, for example. + // + // * After the above two have failed, we just kinda grapple attempting to + // return *something*. + let mut cred_helper = git2::CredentialHelper::new(url); + cred_helper.config(cfg); + let mut cred_error = false; + let ret = f(|url, username, allowed| { + let creds = if allowed.contains(git2::SshKey) { + let user = username.map(|s| s.to_string()) + .or_else(|| cred_helper.username.clone()) + .unwrap_or("git".to_string()); + git2::Cred::ssh_key_from_agent(user.as_slice()) + } else if allowed.contains(git2::UserPassPlaintext) { + git2::Cred::credential_helper(cfg, url, username) + } else if allowed.contains(git2::Default) { + git2::Cred::default() + } else { + Err(git2::Error::from_str("no authentication available")) + }; + cred_error = creds.is_err(); + creds + }); + if cred_error { + ret.chain_error(|| { + human("Failed to authenticate when downloading repository") + }) + } else { + ret + } +} + +fn fetch(repo: &git2::Repository, url: &str) -> CargoResult<()> { + // Create a local anonymous remote in the repository to fetch the url + let refspec = "refs/heads/*:refs/heads/*"; + + with_authentication(url, &try!(repo.config()), |f| { + let mut remote = try!(repo.remote_create_anonymous(url.as_slice(), + refspec)); + try!(remote.add_fetch("refs/tags/*:refs/tags/*")); + remote.set_credentials(f); + try!(remote.fetch(None, None)); + Ok(()) + }) +} diff --git a/src/etc/print-new-snapshot.py b/src/etc/print-new-snapshot.py index 5ddb34bab..ace207052 100644 --- a/src/etc/print-new-snapshot.py +++ b/src/etc/print-new-snapshot.py @@ -22,7 +22,7 @@ snaps = { for platform in sorted(snaps): triple = snaps[platform] tarball = 'cargo-nightly-' + triple + '.tar.gz' - url = 'http://static.rust-lang.org/cargo-dist/' + date + '/' + tarball + url = 'https://static-rust-lang-org.s3.amazonaws.com/cargo-dist/' + date + '/' + tarball dl_path = "target/dl/" + tarball ret = subprocess.call(["curl", "-s", "-o", dl_path, url]) if ret != 0: diff --git a/src/snapshots.txt b/src/snapshots.txt index b4b151e0b..04d4dab04 100644 --- a/src/snapshots.txt +++ b/src/snapshots.txt @@ -1,3 +1,10 @@ +2014-09-03 + linux-i386 d357756680a60cd00464fa991b71170dcddb2b30 + linux-x86_64 35fd121fda3509cc020d42223017be03a1c19b87 + macos-i386 40aad83e9d97f5a344179f4573807f3ac04775f9 + macos-x86_64 5e64f637019f499585ab100e5072b8eeeba191ed + winnt-i386 fc25a2f6f9ce3a6f11348ffe17e1115ca81fc4db + 2014-08-19 linux-i386 8d20fc36b8b7339fcd1ae6c118f1becd001c2b08 linux-x86_64 46e05521f0dceeb831462caa8a54ca1caf21c078 diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 4778abf56..9b0feb865 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -430,7 +430,6 @@ impl<'a> ham::Matcher<&'a [u8]> for ShellWrites { fn matches(&self, actual: &[u8]) -> ham::MatchResult { - println!("{}", actual); let actual = String::from_utf8_lossy(actual); let actual = actual.to_string(); ham::expect(actual == self.expected, actual) diff --git a/tests/test_cargo_build_auth.rs b/tests/test_cargo_build_auth.rs new file mode 100644 index 000000000..3be3f3b97 --- /dev/null +++ b/tests/test_cargo_build_auth.rs @@ -0,0 +1,226 @@ +use std::io::{TcpListener, Listener, Acceptor, BufferedStream}; +use std::io::net::tcp::TcpAcceptor; +use std::collections::HashSet; +use git2; + +use support::{project, execs, ResultTest, UPDATING}; +use support::paths; +use hamcrest::assert_that; + +fn setup() { +} + +struct Closer { a: TcpAcceptor } + +impl Drop for Closer { + fn drop(&mut self) { + let _ = self.a.close_accept(); + } +} + +// Test that HTTP auth is offered from `credential.helper` +test!(http_auth_offered { + let mut listener = TcpListener::bind("127.0.0.1", 0).assert(); + let addr = listener.socket_name().assert(); + let mut a = listener.listen().unwrap(); + let a2 = a.clone(); + let _c = Closer { a: a2 }; + let (tx, rx) = channel(); + + fn headers(rdr: &mut R) -> HashSet { + let valid = ["GET", "Authorization", "Accept", "User-Agent"]; + rdr.lines().map(|s| s.unwrap()) + .take_while(|s| s.len() > 2) + .map(|s| s.as_slice().trim().to_string()) + .filter(|s| { + valid.iter().any(|prefix| s.as_slice().starts_with(*prefix)) + }) + .collect() + } + + spawn(proc() { + let mut s = BufferedStream::new(a.accept().unwrap()); + let req = headers(&mut s); + s.write(b"\ + HTTP/1.1 401 Unauthorized\r\n\ + WWW-Authenticate: Basic realm=\"wheee\"\r\n + \r\n\ + ").unwrap(); + assert_eq!(req, vec![ + "GET /foo/bar/info/refs?service=git-upload-pack HTTP/1.1", + "Accept: */*", + "User-Agent: git/1.0 (libgit2 0.21.0)", + ].move_iter().map(|s| s.to_string()).collect()); + drop(s); + + let mut s = BufferedStream::new(a.accept().unwrap()); + let req = headers(&mut s); + s.write(b"\ + HTTP/1.1 401 Unauthorized\r\n\ + WWW-Authenticate: Basic realm=\"wheee\"\r\n + \r\n\ + ").unwrap(); + assert_eq!(req, vec![ + "GET /foo/bar/info/refs?service=git-upload-pack HTTP/1.1", + "Authorization: Basic Zm9vOmJhcg==", + "Accept: */*", + "User-Agent: git/1.0 (libgit2 0.21.0)", + ].move_iter().map(|s| s.to_string()).collect()); + + tx.send(()); + }); + + let script = project("script") + .file("Cargo.toml", r#" + [project] + name = "script" + version = "0.0.1" + authors = [] + "#) + .file("src/main.rs", r#" + fn main() { + println!("username=foo"); + println!("password=bar"); + } + "#); + assert_that(script.cargo_process("build").arg("-v"), + execs().with_status(0)); + let script = script.bin("script"); + + let config = paths::home().join(".gitconfig"); + let mut config = git2::Config::open(&config).unwrap(); + config.set_str("credential.helper", + script.display().to_string().as_slice()).unwrap(); + + let p = project("foo") + .file("Cargo.toml", format!(r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies.bar] + git = "http://127.0.0.1:{}/foo/bar" + "#, addr.port).as_slice()) + .file("src/main.rs", ""); + + assert_that(p.cargo_process("build").arg("-v"), + execs().with_status(101).with_stdout(format!("\ +{updating} git repository `http://{addr}/foo/bar` +", + updating = UPDATING, + addr = addr, + ).as_slice()) + .with_stderr(format!("\ +Unable to update http://{addr}/foo/bar + +Caused by: + failed to clone into: [..] + +Caused by: + [12] [..] status code: 401 +", + addr = addr))); + + rx.recv(); +}) + +// Boy, sure would be nice to have a TLS implementation in rust! +test!(https_something_happens { + let mut listener = TcpListener::bind("127.0.0.1", 0).assert(); + let addr = listener.socket_name().assert(); + let mut a = listener.listen().unwrap(); + let a2 = a.clone(); + let _c = Closer { a: a2 }; + let (tx, rx) = channel(); + spawn(proc() { + drop(a.accept().unwrap()); + + tx.send(()); + }); + + let p = project("foo") + .file("Cargo.toml", format!(r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies.bar] + git = "https://127.0.0.1:{}/foo/bar" + "#, addr.port).as_slice()) + .file("src/main.rs", ""); + + assert_that(p.cargo_process("build").arg("-v"), + execs().with_status(101).with_stdout(format!("\ +{updating} git repository `https://{addr}/foo/bar` +", + updating = UPDATING, + addr = addr, + ).as_slice()) + .with_stderr(format!("\ +Unable to update https://{addr}/foo/bar + +Caused by: + failed to clone into: [..] + +Caused by: + [[..]] {errmsg} +", + addr = addr, + errmsg = if cfg!(windows) { + "Failed to send request: The connection with the server \ + was terminated abnormally\n" + } else { + "SSL error: [..]" + }))); + + rx.recv(); +}) + +// Boy, sure would be nice to have an SSH implementation in rust! +test!(ssh_something_happens { + let mut listener = TcpListener::bind("127.0.0.1", 0).assert(); + let addr = listener.socket_name().assert(); + let mut a = listener.listen().unwrap(); + let a2 = a.clone(); + let _c = Closer { a: a2 }; + let (tx, rx) = channel(); + spawn(proc() { + drop(a.accept().unwrap()); + + tx.send(()); + }); + + let p = project("foo") + .file("Cargo.toml", format!(r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies.bar] + git = "ssh://127.0.0.1:{}/foo/bar" + "#, addr.port).as_slice()) + .file("src/main.rs", ""); + + assert_that(p.cargo_process("build").arg("-v"), + execs().with_status(101).with_stdout(format!("\ +{updating} git repository `ssh://{addr}/foo/bar` +", + updating = UPDATING, + addr = addr, + ).as_slice()) + .with_stderr(format!("\ +Unable to update ssh://{addr}/foo/bar + +Caused by: + failed to clone into: [..] + +Caused by: + [23] Failed to start SSH session: Failed getting banner +", + addr = addr))); + + rx.recv(); +}) diff --git a/tests/tests.rs b/tests/tests.rs index 95ef99315..cc96d9ae1 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -1,9 +1,10 @@ #![feature(macro_rules)] #![feature(phase)] -extern crate term; extern crate cargo; +extern crate git2; extern crate hamcrest; +extern crate term; extern crate url; #[phase(plugin, link)] @@ -39,3 +40,4 @@ mod test_cargo_freshness; mod test_cargo_generate_lockfile; mod test_cargo_profiles; mod test_cargo_package; +mod test_cargo_build_auth; -- 2.30.2